from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time

# getResultsAndComputePAt10()
# generateRandomDocids()

def getResultsAndComputePAt10():
    numOfDocumentsInTotalInGov2 = 25205179
    numOfDocsAtDebugPercent = 10
    numOfDocsAt1Percent = int(numOfDocumentsInTotalInGov2 * 0.01)
    numOfDocsAt3Percent = int(numOfDocumentsInTotalInGov2 * 0.03)
    numOfDocsAt5Percent = int(numOfDocumentsInTotalInGov2 * 0.05)
    numOfDocsAt10Percent = int(numOfDocumentsInTotalInGov2 * 0.1)
    numOfDocsAt15Percent = int(numOfDocumentsInTotalInGov2 * 0.15)
    numOfDocsAt20Percent = int(numOfDocumentsInTotalInGov2 * 0.2)
    numOfDocsAt30Percent = int(numOfDocumentsInTotalInGov2 * 0.3)
    numOfDocsAt40Percent = int(numOfDocumentsInTotalInGov2 * 0.4)
    numOfDocsAt50Percent = int(numOfDocumentsInTotalInGov2 * 0.5)
    print "numOfDocsAt1Percent:",numOfDocsAt1Percent
    print "numOfDocsAt3Percent:",numOfDocsAt3Percent
    print "numOfDocsAt5Percent:",numOfDocsAt5Percent
    print "numOfDocsAt10Percent:",numOfDocsAt10Percent
    print "numOfDocsAt15Percent:",numOfDocsAt15Percent
    print "numOfDocsAt20Percent:",numOfDocsAt20Percent
    print "numOfDocsAt30Percent:",numOfDocsAt30Percent
    print "numOfDocsAt40Percent:",numOfDocsAt40Percent
    print "numOfDocsAt50Percent:",numOfDocsAt50Percent

    inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentsSelectedAtDifferentPercentage_20140412"
    inputFileHanlder = open(inputFileName1,"r")
    currentLine = inputFileHanlder.readline()

    docIDDict = {}
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocIDIndex = currentLineElements[0]
        currentDocID = currentLineElements[1]
        if currentDocID not in docIDDict:
            docIDDict[currentDocID] = 1
        else:
            print "duplicated docID detected."
            exit(1)
        if len(docIDDict) == numOfDocsAt1Percent:
            break
        currentLine = inputFileHanlder.readline()
    print "len(docIDDict):",len(docIDDict)
    inputFileHanlder.close()

    # key: qID
    # value: list of document candidates
    qIDWithTOP10DocumentResultDict = {}
    inputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/rawResultFile_150_human_queries_top2M_AND_20140412"
    inputFileHanlder2 = open(inputFileName2,"r")
    currentQueryID = ""
    currentLine = inputFileHanlder2.readline()
    while currentLine:
        if currentLine.strip().startswith("qid:"):
            currentLineElements = currentLine.strip().split(" ")
            print currentLineElements
            exit()
            currentQueryID = currentLineElements[1]
            
            # for debug
            if currentQueryID == "702":
                break
            
            if currentQueryID not in qIDWithTOP10DocumentResultDict:
                qIDWithTOP10DocumentResultDict[currentQueryID] = []
        if currentLine.strip().startswith("Score:"):
            currentLineElements = currentLine.strip().split("\t")
	    currentDocIDStringPair = currentLineElements[1]
 	    currentDocID = currentDocIDStringPair.strip().split(":")[1].strip()
	    print currentDocID
        
        if currentDocID not in docIDDict:
            pass
        else:
            if len(qIDWithTOP10DocumentResultDict[currentQueryID]) <= 10:
                pass
            else:
                qIDWithTOP10DocumentResultDict[currentQueryID].pop()
            qIDWithTOP10DocumentResultDict[currentQueryID].append(currentDocID)
            
        currentLine = inputFileHanlder2.readline()
    print "len(qIDWithTOP10DocumentResultDict):",len(qIDWithTOP10DocumentResultDict)
    # print "qIDWithTOP10DocumentResultDict:",qIDWithTOP10DocumentResultDict
    inputFileHanlder2.close()

def generateRandomDocids():
    print "Begins..."
    # for moa:
    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/randomDocumentsSelectedAtDifferentPercentage_20140412"
    outputFileHandler = open(outputFileName,"w")
    numOfDocumentsInTotalInGov2 = 25205179
    docCollectionLowerBound = 0
    docCollectionUpperBound = 25205178
    numOfDocsAtDebugPercent = 10
    numOfDocsAt1Percent = int(numOfDocumentsInTotalInGov2 * 0.01)
    numOfDocsAt3Percent = int(numOfDocumentsInTotalInGov2 * 0.03)
    numOfDocsAt5Percent = int(numOfDocumentsInTotalInGov2 * 0.05)
    numOfDocsAt10Percent = int(numOfDocumentsInTotalInGov2 * 0.1)
    numOfDocsAt15Percent = int(numOfDocumentsInTotalInGov2 * 0.15)
    numOfDocsAt20Percent = int(numOfDocumentsInTotalInGov2 * 0.2)
    numOfDocsAt30Percent = int(numOfDocumentsInTotalInGov2 * 0.3)
    numOfDocsAt40Percent = int(numOfDocumentsInTotalInGov2 * 0.4)
    numOfDocsAt50Percent = int(numOfDocumentsInTotalInGov2 * 0.5)
    print "numOfDocsAtDebugPercent:",numOfDocsAtDebugPercent
    print "numOfDocsAt1Percent:",numOfDocsAt1Percent
    print "numOfDocsAt3Percent:",numOfDocsAt3Percent
    print "numOfDocsAt5Percent:",numOfDocsAt5Percent
    print "numOfDocsAt10Percent:",numOfDocsAt10Percent
    print "numOfDocsAt15Percent:",numOfDocsAt15Percent
    print "numOfDocsAt20Percent:",numOfDocsAt20Percent
    print "numOfDocsAt30Percent:",numOfDocsAt30Percent
    print "numOfDocsAt40Percent:",numOfDocsAt40Percent
    print "numOfDocsAt50Percent:",numOfDocsAt50Percent

    docIDDict = {}
    docIDList = []

    while True:
        currentDocIDSelected = random.randint(docCollectionLowerBound, docCollectionUpperBound)
        if currentDocIDSelected not in docIDDict:
            docIDDict[currentDocIDSelected] = 1
            docIDList.append(currentDocIDSelected)
        if len(docIDDict) == numOfDocsAtDebugPercent:
            break

    for index,docID in enumerate(docIDList):
        outputFileHandler.write(str(index) + " " + str(docID) + "\n")
    print "Overall:"
    print "outputFileName:",outputFileName
    outputFileHandler.close()
    print "Ends."

getResultsAndComputePAt10()



























